<?php
class rssCollector{
    
	//新闻来源
    public $rssSourceMap = array();
	//分类映射表
	public $categoryMap = array();
    //上次抓取时间
    public $lastGetTime = '';
    //数据存储区
	public $dataPath = '';
	
	//总时间
    protected $totalTime = 0;
    //网络传输耗费时间
    protected $transferTime = 0;
    //正则表达式解析时间
    protected $parseTime = 0;
    //入库时间
    protected $saveTime = 0;
	
	//成功抓取总数
    protected $successCount = 0;
	//忽略条数
	protected $ignoreCount = 0;
	//内容匹配失败条数
    protected $matchErrorCount = 0;
    //保存失败条数
	protected $saveErrorCount = 0;
	
    /**
     * 执行抓取
     */
    public function execut(){
		
	}
    
    /**
     * 将XML转换为数组形式
     * @param type $xmlSource
     * @return type
     */
    protected function convertXMLToArray($xmlSource){
        $simpleXml = simplexml_load_string($xmlSource, 'SimpleXMLElement', LIBXML_NOCDATA);
        $targetArr = json_decode(json_encode($simpleXml), true);
        return $targetArr;
    }
	
	/**
     * 简单请求
     * @param $url type string
     * @return type string
     */
	protected function simpleRequest($url){
		$ch = curl_init();
		$options = array(
			CURLOPT_URL => $url,
            CURLOPT_FOLLOWLOCATION => true, 
            CURLOPT_AUTOREFERER => true,
            //不输出头部
            CURLOPT_HEADER => false, 
            //输出body
            CURLOPT_NOBODY => false, 
            CURLOPT_RETURNTRANSFER => true,
            //不使用原生输出
            CURLOPT_BINARYTRANSFER => false, 
            CURLOPT_HTTPGET => true,
			//最大执行时间
			CURLOPT_TIMEOUT => 10
        );
		curl_setopt_array($ch, $options);
		$res = curl_exec($ch);
		curl_close($ch);
		return $res;
	}

	/**
     * 多线程并发请求
     * @param type $nodes  请求节点 形如array('sina' => 'http://rss.sina.com.cn/', 'soho' => 'http://www.sohu.com/')
     * @return type
     */
    protected function multipleThreadsRequest($nodes){ 
        $options = array(
            CURLOPT_FOLLOWLOCATION => true, 
            CURLOPT_AUTOREFERER => true,
            //不输出头部
            CURLOPT_HEADER => false, 
            //输出body
            CURLOPT_NOBODY => false, 
            CURLOPT_RETURNTRANSFER => true,
            //CURLOPT_REFERER => 'www.baidu.com', 
            //不使用原生输出
            CURLOPT_BINARYTRANSFER => false, 
            CURLOPT_HTTPGET => true,
			//最大执行时间
			CURLOPT_TIMEOUT => 10
        );
        //初始化多线程curl句柄
        $mh = curl_multi_init();
        //根据节点实例化请求池
        $curl_array = array(); 
        foreach($nodes as $i => $url){ 
            $curl_array[$i] = curl_init($url); 
            curl_setopt_array($curl_array[$i], $options);
            curl_multi_add_handle($mh, $curl_array[$i]); 
        }
        //执行请求池里所有请求
        $running = NULL; 
        do {
            curl_multi_exec($mh,$running); 
        } while($running > 0);
        //获取内容
        $res = array(); 
        foreach($nodes as $i => $url) { 
            $res[$i] = curl_multi_getcontent($curl_array[$i]); 
        } 
        //释放资源
        foreach($nodes as $i => $url){ 
            curl_multi_remove_handle($mh, $curl_array[$i]); 
        } 
        curl_multi_close($mh);
        //返回结果
        return $res; 
    }
	
	//获取制定标签内容
	protected function matchDiv($content, $div_pattern){
		$i = $j = $pos = 0;
		$arr = $matchs = array();
		if(!preg_match($div_pattern, $content, $matchs)){
			return false;
		}
		$div = $matchs[0];
		$contentStr = str_replace($div, '', strstr($content, $div)) ;
		$arr[] = '1';
		while(1){
			$i = strpos($contentStr, '</div>', $i);
			//不能正确匹配
			if($i === false){
				$pos = 0;
				break;
			}
			$t = $j;
			$j = strpos($contentStr, '<div', $j);
			if($j === false || $i < $j){
				$j = $t;
				array_pop($arr);
				$i++;
			}else{
				$arr[] = '1';
				$j++;
			}
			if(count($arr) == 0){
				$pos = --$i;
				break;
			}
		}
		return substr($contentStr, 0, $pos);
	}
	
	//把图片地址从相对地址修改为绝对地址
	protected function imgRelToAbs($content, $url){
		if(!$content || !$url){
			return false;
		}
		//匹配出相对当前url的图片路径的前缀
		preg_match('/.*\//i', $url, $matches);
		$urlDirCurrent = isset($matches[0]) ? $matches[0] : '';
		//匹配出相对根目录的图片地址前缀
		preg_match('/http:\/\/.*?\//i', $url, $matches);
		$urlDirRoot = isset($matches[0]) ? substr($matches[0], 0, -1) : '';
		$imgs = $this->matchImgs($content);
		foreach ($imgs as $value) {
			if(strpos($value, 'http://') === 0){//绝对图片地址
				continue;
			}elseif(strpos($value, '/') === 0){//相对于根目录
				$content = str_replace($value, $urlDirRoot.$value, $content);
			}else{//相对于当前网页
				$content = str_replace($value, $urlDirCurrent.$value, $content);
			}
		}
		return $content;
	}
	
	//获取给定HTML片断里所有图片地址
	protected function matchImgs($content){
		$parten = '/<img(?:.*?)src="(.*?)"/i';
		preg_match_all($parten, $content, $matches);
		return isset($matches[1]) ? $matches[1] : array();
	}
	
	//保存至数据库
    protected function saveToDB($newsArr){
		foreach ($newsArr as $news){
			try {
				$sql = "INSERT INTO content_temp(news_id, source, source_url, category, channel, author, title, keywords, description, content, slides, video, img, imgs, publishDate, publishTime, status) VALUES(:news_id, :source, :source_url, :category, :channel, :author, :title, :keywords, :description, :content, :slides, :video, :img, :imgs, :publishDate, :publishTime, :status)";
				$data['news_id'] = $news['news_id'];
				$data['source'] = $news['source'];
				$data['source_url'] = $news['link'];
				$data['category'] = $news['category'];
				$data['channel'] = $news['channel'];
				$data['author'] = $news['author'];
				$data['title'] = $news['title'];
				$data['keywords'] = $news['keywords'];
				$data['description'] = $news['description'];
				$data['content'] = $news['content'];
				$data['slides'] = $news['slides'];
				$data['video'] = $news['video'];
				$data['img'] = $news['img'];
				$data['imgs'] = $news['imgs'];
				$data['publishDate'] = date('Y-m-d', $news['pubDate']);
				$data['publishTime'] = $news['pubDate'];
				$data['status'] = '0';
				DB::insert($sql, $data);
				$this->successCount += 1;
			} catch (PDOException $ex) {
				if('23000' != $ex->getCode()){
					$this->saveErrorCount += 1;
				}
				continue;
			}
		}
    }
	
	//获取RSS源
    public function getCategory(){
		$data = require $this->dataPath;
		return $data['category'];
    }
	
	//获取分类映射表
	public function getCategoryMap(){
		$data = require $this->dataPath;
		return $data['map'];
	}
	
	//通过映射表获取给定新华网分类对应的乐媒分类
	public function getLocalCategory($categoryName, $default = 1){
		return isset($this->categoryMap[$categoryName]) ? $this->categoryMap[$categoryName] : $default;
	}
	
	//修改新华网分类映射
	public function changeCatgoryMap($map){
		$data = require $this->dataPath;
		$data['map'] = $map;
		$data_str = '<?php return '.var_export($data, true).';';
        return file_put_contents($this->dataPath, $data_str);
	}
	
	//获取上次抓取时间
    protected function getLastGetTime(){
		$data = require $this->dataPath;
		return $data['last_get_time'];
    }
    
    //设置上次抓取时间为现在时间
    protected function setLastGetTime(){
		$data = require $this->dataPath;
		$data['last_get_time'] = time();
		$data_str = '<?php return '.var_export($data, true).';';
        file_put_contents($this->dataPath, $data_str);
    }
	
	//获取总时间
	public function getTotalTime(){
		return $this->totalTime;
	}
	
	//获取网络传输时间
	public function getTransferTime(){
		return $this->transferTime;
	}
	
	//获取内容解析时间
	public function getParseTime(){
		return $this->parseTime;
	}
	
	//获取入库时间
	public function getSaveTime(){
		return $this->saveTime;
	}
	
	//获取成功条数
	public function getSuccessCount(){
		return $this->successCount;
	}
	
	//获取忽略条数
	public function getIgnoreCount(){
		return $this->ignoreCount;
	}
	
	//获取匹配失败条数
	public function getMatchErrorCount(){
		return $this->matchErrorCount;
	}
	
	//获取保存失败条数
	public function getSaveErrorCount(){
		return $this->saveErrorCount;
	}
	
	//获取统计数据
	public function getInfo(){
		return array(
			'totalTime' => (int)$this->getTotalTime(),
			'transferTime' => (int)$this->getTransferTime(),
			'parseTime' => (int)$this->getParseTime(),
			'saveTime' => (int)$this->getSaveTime(),
			'successCount' => $this->getSuccessCount(),
			'ignoreCount' => $this->getIgnoreCount(),
			'matchErrorCount' => $this->getMatchErrorCount(),
			'saveErrorCount' => $this->getSaveErrorCount()
		);
	}
}
